# encoding: utf-8
import sys, os
import sys

import json
import shutil
import codecs
import time
import datetime
# 多线程处理svn信息
import threading
from collections import OrderedDict
import subprocess
import os.path
# 保存数组到txt
import numpy as np

from types import TracebackType
from typing import Optional
from PyQt5.QtWidgets import QFileDialog, QListWidget,QFrame, QMainWindow, QWidget, QAbstractItemView,QMessageBox, QListWidgetItem,QDialog,QApplication
from PyQt5.QtGui import QDragEnterEvent, QDropEvent, QDragMoveEvent, QKeyEvent,QStandardItem,QStandardItemModel
from PyQt5.QtCore import QObject, Qt, QModelIndex
from PyQt5 import QtCore, QtGui, QtWidgets
from resource.UI.Ui_main import Ui_Dialog as mainDialog 
# import script.uilogic.DirtoexcelLogic as dirlogic
# 处理下载页面
import urllib
import requests
import re
from urllib.parse import urlparse
import importlib
importlib.reload(sys)
from bs4 import BeautifulSoup
# 重定向控制台的输出
#控制台
class EmittingStream(QtCore.QObject):
  textWritten = QtCore.pyqtSignal(str)  #定义一个发送str的信号
  def write(self, text):
      self.textWritten.emit(str(text))

# 主窗口逻辑
class MainDialog(QDialog,mainDialog):
  
  def __init__(self):
    super(MainDialog, self).__init__()
    
    self.setWindowFlags(Qt.WindowMinMaxButtonsHint|Qt.WindowCloseButtonHint)
    self.setupUi(self)
    # 测试数据
    self.lineEdit_url.setText("https://meyoonet.com/")
    #下面将输出重定向到textEdit中
    sys.stdout = EmittingStream(textWritten=self.outputWritten)
    sys.stderr = EmittingStream(textWritten=self.outputWritten)
    self.textEdit_console.setReadOnly(True)
    # 添加按钮时间
    self.btn_choosefile.clicked.connect(self.chooseSaveFile)
    # 选择useragent
    self.comboBox_uage.currentIndexChanged.connect(self.setUseragent)
    #开始下载
    self.btn_startdownload.clicked.connect(self.startDownload)
    
  def chooseSaveFile(self):
    print("选择文件夹")
    directory = QtWidgets.QFileDialog.getExistingDirectory(None,"选取文件夹",'')
    self.lineEdit_savedir.setText(directory)
  def setUseragent(self):
    try:
      if self.comboBox_uage.currentIndex() == None or self.comboBox_uage.currentIndex() == 0:
        self.lineEdit_useragent.setText('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36')
      if self.comboBox_uage.currentIndex() == 1:
        self.lineEdit_useragent.setText('Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1')
        # self.label_3.setText(str(self.name_money[self.comboBox_uage.itemText(self.comboBox_uage.currentIndex())]))
    except Exception as e:
      pass
  def startDownload(self):
    
    url = self.lineEdit_url.text()
    if url == '':
      QMessageBox.information(self,"提示","您输入的网址为空")
      return
    print('开始下载>>>')
    useragenttxt = self.lineEdit_useragent.text()
    cookietxt = self.lineEdit_cookie.text()
    self.timeout = self.lineEdit_time.text();
    self.savedir = self.lineEdit_savedir.text();
    self.headers = {   
                    "Cookie": cookietxt,
                    'User-Agent': useragenttxt
                    }
    print(self.headers)
    self.hand(url)

  def hand(self,url):
    # 请求网站
    req = requests.get(url, headers=self.headers)
    req.encoding = req.apparent_encoding
    status = req.status_code
    if status == 200:
        print ("HostUrl:"+url)
        # 解析网址
        urlparses = urlparse(url)
        # 提取网址
        baseurl = urlparses.netloc
        # 网页前缀
        scheme = urlparses.scheme
        print(urlparses)
        # 给本连接明名
        htmlname = self.htmlName(url)
        # 创建文件夹
        print('创建目录>>>')
        self.mkprojectdir(self.savedir + '/' +baseurl)
        self.progressBar.setValue(10)
        self.progressBar.setValue(20)
        # 获取soup
        # print(req.text)
        soup = BeautifulSoup(req.text,"html.parser")
        self.progressBar.setValue(30)
        newhtml = self.savescript(scheme,baseurl,soup,req.text)
        print('加载js>>>')
        self.progressBar.setValue(40)
        newhtml = self.savePics(baseurl,soup,newhtml,scheme)
        print('保存图片>>>')
        self.progressBar.setValue(50)
        self.progressBar.setValue(60)
        newhtml = self.savecss(scheme,baseurl,soup,newhtml)
        print('保存css文件>>>')
        self.progressBar.setValue(70)
        allurl = scheme+"://"+baseurl+"/"
        newhtml = newhtml.replace('href="/','href="').replace('src="/','src="').replace(allurl,'')
        self.progressBar.setValue(80)
        self.progressBar.setValue(90)
        print('保存html文件')
        self.writedoc(baseurl,htmlname,newhtml)
        self.progressBar.setValue(100)
    else:
        print ("HostUrl:"+url)
  def savecss(self,scheme,baseurl,soup,html):
    links = soup.find_all(rel="stylesheet")
    for link in links:
        css = link.get("href")
        if css:
            if "http" in css:
                re_css = css
            else:
                re_css= scheme+"://"+baseurl+"/"+css
                re_css = re_css.replace("..","")
                # re_css = re_css.replace("//","/")
            x= css.split('/')
            ncss = "/css/"+x[len(x)-1]
            vcss = re.findall(r"(.*)\?", ncss)
            # CSS地址
            bacecssurl = re_css.replace(x[len(x)-1],"")
            if vcss:
              ncss = vcss[0]
              
            try:
              cssinfo = requests.get(re_css, headers=self.headers)
              cssinfo.encoding = cssinfo.apparent_encoding
              # 打开一个文件
              fo = open(self.savedir + '/' + baseurl+ncss, "w+",encoding='utf-8')
              csshtml = cssinfo.text
              bkimgs = re.findall(r"background-image: url\((.*)\);", csshtml)
              if bkimgs:
                  for bkimg in bkimgs:
                      f= bkimg.split('/')
                      nimg = "../images/"+f[len(f)-1]
                      if "http" in bkimg:
                          re_img = bkimg
                      else:
                          simg = bkimg.replace("../","/")
                          ssimg = simg.replace("//","/").replace("///","/")
                          if "/" not in ssimg:
                              re_img= bacecssurl+"/"+ssimg
                          else:
                              re_img= scheme+"://"+baseurl+ssimg
                      try:
                          newimgsrc = baseurl+'/images/%s' % f[len(f)-1]
                          urllib.urlretrieve(re_img,newimgsrc)
                          csshtml = csshtml.replace(bkimg,nimg)
                      except:
                          errorimg = "Img Save Error"
              fo.write(csshtml)
              html = html.replace(css,ncss)
              # 关闭打开的文件
              fo.close()
            except:
                continue
    return html
  def savescript(self,scheme,baseurl,soup,html):
    scripts = soup.find_all("script")
    for script in scripts:
        js = script.get('src')
        if js:
            if "http" in js:
                re_js = js
            else:
                re_js = scheme+"://"+baseurl+"/"+js
                re_js = re_js.replace("..","")
                # re_js = re_js.replace("//","/")
            x= js.split('/')
            njs = "/js/"+x[len(x)-1]
            vjs = re.findall(r"(.*)\?", njs)
            if vjs:
                njs = vjs[0]
            try:
                jsinfo = requests.get(re_js, headers=self.headers)
                # 打开一个文件
                fo = open(self.savedir + '/' +baseurl+njs, "w+",encoding='utf-8')
                fo.write(jsinfo.text)
                html = html.replace(js,njs)
                # 关闭打开的文件
                fo.close()
                html.replace(js,njs)
            except:
                continue
    return html
  def htmlName(self,requrl):
      resurl = requrl.replace('\t','').replace('\n','').replace(' ','')
      x= resurl.split('/')

      ncss = x[len(x)-1]
      vcss = re.findall(r"(.*)\?", ncss)
      if vcss:
          ncss = vcss[0]

      if ncss:
          if ".html" in ncss:
              return ncss
          else:
              if ".htm" in ncss:
                  return ncss
              else:
                  return ncss+".html"
      else:
          return "index.html"

  def writedoc(self,baseurl,filename,content):
      # 打开一个文件
      fo = open(self.savedir + '/' +baseurl+"/"+filename, mode="w+",encoding='utf-8')
      # s.decode('gbk', 'ignore').encode('utf-8') 
      fo.write(content)

      # 关闭打开的文件
      fo.close()

  def savePics(self,baseurl,soup,text,scheme):
      allimgs = soup.find_all('img')
      html = text
      for allimg in allimgs:
          link = allimg.get('src')
          if "http" in link:
              newlink = link
          else:
              newlink = scheme+"://"+baseurl+'/'+link
          print(newlink)
          x= link.split('/')
          nimg = "images/"+x[len(x)-1]
          vimg = re.findall(r"(.*)\?", nimg)
          if vimg:
              nimg = vimg[0]
          # 测试下载图片
          try:
              newimgsrc = self.savedir + '/' +baseurl+'/%s' % nimg
              print('下载：'+ newimgsrc)
              urllib.request.urlretrieve(newlink,newimgsrc)
              html = html.replace(link,nimg)
          except:
              errorimg = "Img Save Error"
      return html


  def mkprojectdir(self,baseurl):
   
    exists = os.path.exists(baseurl)
    if exists:
        print ("Notice:"+"目录已经存在")
    else:
        os.mkdir(baseurl)
        os.mkdir(baseurl+"/css")
        os.mkdir(baseurl+"/js")
        os.mkdir(baseurl+"/images")

  def jindu(self,n):
      import sys,time
      fmt = '{:3d} [{:<100}]'.format
      print ('\r'+fmt(n, '='*n))
      
  def outputWritten(self, text):
    cursor = self.textEdit_console.textCursor()
    cursor.movePosition(QtGui.QTextCursor.End)
    # text = '[' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ']' + text+'\n'
    cursor.insertText(text)
    self.textEdit_console.setTextCursor(cursor)
    self.textEdit_console.ensureCursorVisible()
if __name__ == '__main__':
    app = QApplication(sys.argv)
    MainWindow = MainDialog()
    MainWindow.show()
    sys.exit(app.exec_())
